/**
 * 北谷信息科技（广州）有限公司拥有本软件所有版权 © 2022，并保留所有权利。
 * Copyright © 2022, North Valley Information Technology (Guangzhou) Company
 * Limited, All Rights Reserved.
 */

package main.com.scrapy;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import main.com.scrapy.entity.ProductParam;
import main.com.scrapy.utils.HttpUtil;
import main.com.scrapy.utils.JsonUtil;
import org.jsoup.nodes.Document;

/**
 * 抓取启动类
 *
 * @author Double
 * <p><b>Creation Time:</b> 2022/9/17 10:43</p>
 * @since scrapy-1688
 */
public class ScrapyMain {


  public static void main(String[] args) throws Exception {
    // 读取execl参数，根据参数进行数据爬取，若参数为空不操作
    //    File file = new File("D:/ideaproject/input.xlsx");
    //
    //    System.out.println("文件路径" + file.getAbsolutePath());
    //    ExcelUtil.readExcel(file);

    // 抓取所有商铺信息
    //    getAllCompany();
    //    getAllCompany2();
    // 获取商铺查询条件，关键字和省市查询
    //    selectCompany("keywords", "province", "city");

    // 若商铺链接不为空，导出该商铺所有商品链接到production-link文件下，以商铺id/商品类目.xlsx分类存储链接
    getProductLinks();

    // 选择商品链接文件导出商品的详细信息，每条sku信息详细列出
    // https://detail.1688.com/offer/678975545441.html
    //    String productId = "681161690505";
    //    getProductDetail(productId);

    // 按商品关键字获取商品，按商品销量排序，并导出商品链接、名称和销量
    // https://fsyj18.1688.com/page/offerlist.html?keywords=%E4%BC%91%E9%97%B2

  }

  /**
   * TODO:模拟登录获取cookie、调用接口RGV587_ERROR::SM::哎哟喂,被挤爆啦,请稍后重试的响应报错
   */
  private static void getProductLinks() throws Exception {
    // 商铺所有商品静态页面链接 https://fsyj18.1688.com/page/offerlist.htm
    //    String companyUrl = "http://shop1l9f891648338.1688.com";
    //    String productUrl = companyUrl + "/page/offerlist.htm";
    //    Document productDoc = HttpUtil.getHttpDocument(productUrl);

    // 动态页面的数据获取 - 访问页面调用接口获取数据再渲染页面
    String productApi = "https://h5api.m.1688.com/h5/mtop.1688.shop.data.get/1.0/";
    ProductParam param = new ProductParam();

    String cookie = "cna=6Uu1GysvdEUCATo+ps2t/ddW; cookie2=1ab1829e437e6417fdb8fac6b7146d9e; t=d6ca2d037d2e80d8d749947162efb60e; _tb_token_=f41eefe91b575; xlly_s=1; keywordsHistory=男童衬衣; ali_ab=58.62.166.205.1663984130983.7; alicnweb=touch_tb_at=1663987764616; _m_h5_tk=dd6b9e185837584124808c24f78b6286_1664017549677; _m_h5_tk_enc=23d823f44baba79bc177bcb831b37913; cookie1=B0TxOpjr2JCF+44+OeKepZKUpHkbqvsbHpjUfIhTUvo=; cookie17=UU6oIK+/RH65Cg==; sgcookie=E100FD42OAkRZHXHmx79pD121W0tYO9guJwDKIa9D813mcpCEuuYoXbnru4yHm2Ee3KRQQ+cL39HxwmyA0zJx/fkYwdvc8inbHM8Z1GGv6I4Uoc=; sg=105; csg=391693b0; lid=火影340361; unb=2699915560; uc4=nk4=0@2h3pD0fUr2ST4Nrg2MozC0XhMzsw&id4=0@U2xlrqyvlIk344O0AhYmgrh1dgQn; __cn_logon__=true; __cn_logon_id__=火影340361; ali_apache_track=c_mid=b2b-2699915560d7b56|c_lid=火影340361|c_ms=1; ali_apache_tracktmp=c_w_signed=Y; _nk_=\\u706B\\u5F71340361; last_mid=b2b-2699915560d7b56; _csrf_token=1664008513005; is_identity=buyer; _is_show_loginId_change_block_=b2b-2699915560d7b56_false; _show_force_unbind_div_=b2b-2699915560d7b56_false; _show_sys_unbind_div_=b2b-2699915560d7b56_false; _show_user_unbind_div_=b2b-2699915560d7b56_false; __rn_alert__=false; aliwwLastNum=0; l=eBxB60_VTrIai0v3BO5ZKurza779NIdf1sPzaNbMiInca6tlF3KiWNCEYHv6cdtjgt5xVetzoWbIjdHkPlaU-EZ_bKlp3EzK6Kp68e1..; isg=BC8v6s1fkIjMDJS2ScQJJSmdvkM51IP2ExQd4kG9ah71kEySSaB7R6IGEoCu6Ftu; tfstk=cYZhBg1XpyuCKCs81WiC45ENH5xlZ5LEMuz4b9efcJCxRoqNiaYwuDQOIYCfTU1..";
    Map<String, String> kvs = new HashMap<>();
    String[] keyValues = cookie.split("; ");
    Arrays.stream(keyValues).forEach(t -> {
      int index = t.indexOf("=");
      kvs.put(t.substring(0, index), t.substring(index + 1, t.length()));
    });
    String data = "{\"dataType\":\"moduleData\",\"argString\":\"{\\\"memberId\\\":\\\"b2b-2207518548363bd361\\\",\\\"appName\\\":\\\"pcmodules\\\",\\\"resourceName\\\":\\\"wpOfferColumn\\\",\\\"type\\\":\\\"view\\\",\\\"version\\\":\\\"1.0.0\\\",\\\"appdata\\\":{\\\"sortType\\\":\\\"wangpu_score\\\",\\\"sellerRecommendFilter\\\":false,\\\"mixFilter\\\":false,\\\"tradenumFilter\\\":false,\\\"quantityBegin\\\":null,\\\"keywords\\\":\\\"长裤\\\",\\\"pageNum\\\":1,\\\"count\\\":30}}\"}";
    //    params.put("data", JSON.toJSONString(param));
    //    long timestamp = System.currentTimeMillis();
    long timestamp = 1664008532449L;
    Map<String, Object> map = new LinkedHashMap<>();
    map.put("jsv", "2.6.2");
    map.put("appKey", "12574478");
    map.put("t", timestamp);
    map.put("sign", HttpUtil.createSign(cookie, timestamp, data)); // 生成签名
    map.put("api", "mtop.1688.shop.data.get");
    map.put("v", "1.0");
    map.put("type", "json");
    map.put("valueType", "string");
    map.put("dataType", "json");
    map.put("timeout", 10000);

    Map<String, String> params = new HashMap<>();
    params.put("data", data);
    //    Document document = HttpUtil.getHttpDocument(productApi, map, params, cookie);
    Document document = HttpUtil.getHttpDocument2(productApi, map, params, kvs);

    System.out.println("======完成商品链接导出========");
    //    Map<String, Object> selecWtParams = new HashMap<>();
    //    Element dev = productDoc.getElementById("container");
    //    String attr = dev.attr("data-view-config");
    //    JSONObject jsonObject = JSON.parseObject(attr);
    //    Object uid = jsonObject.get("uid");
    //    selectParams.put("keywords", "");
    //    String getParams = HttpUtil.createParams(selectParams);
  }

  private static void getProductDetail(String productId) throws Exception {
    String productDetailUrl = "https://detail.1688.com/offer/";
    Document document = HttpUtil.getHttpDocument(productDetailUrl + productId + ".html");
    String dataRegex = "window.__INIT_DATA=(.+)";
    List<String> jsonResult = JsonUtil.getJsonResult(document.data(), dataRegex);

    String uriRegex = "\\{\"([a-zA-Z_]+)\":(.+)}";
    List<JSONObject> productDetail = new ArrayList<>();
    List<String> jsonRes = JsonUtil.getJsonResult(jsonResult.get(0), uriRegex);
    JSONObject response = JSON.parseObject(jsonRes.get(0));
    JSONObject resData = JSON.parseObject(response.get("globalData").toString());

    System.out.println("=======完成商品详情导出=======");
  }

  // https://search.1688.com/service/companySearchBusinessService?
  // https://s.1688.com/company/company_search.htm=undefined&beginPage=2&async=true&asyncCount=6&keywords=&pageSize=20&requestId=C72zdG2eAXMyFJb5TkMnYimnzyaN8PGkS7d1664011355922&sessionId=ac6be187a7564799ba8d7ca87fbb5fbc&startIndex=0&pageName=supplier&_bx-v=1.1.20
  private static void getAllCompany2() throws Exception {
    Document document = HttpUtil.getHttpDocument(HttpUtil.COMPANY_URL, null, null, null);
    String dataRegex = "window.data.offerresultData = successDataCheck(.+)";
    List<String> jsonResult = JsonUtil.getJsonResult(document.data(), dataRegex);

    // 获取最大页码
    String regex = "\"pageCount\":(\\d+)";
    List<String> countRes = JsonUtil.getJsonResult(jsonResult.get(0), regex);
    String[] countData = countRes.get(0).split(":");
    int pageSize = Integer.parseInt(countData[1]);

    String productApi = "https://search.1688.com/service/companySearchBusinessService";
    Map<String, Object> pageMap = new LinkedHashMap<>();
    pageMap.put("https://s.1688.com/company/company_search.htm", "undefined");
    pageMap.put("async", true);
    pageMap.put("asyncCount", 10);
    //    pageMap.put("keywords", 1);
    pageMap.put("pageSize", 20);
    pageMap.put("requestId", "RwQcK8YHSAWebn3JJe67yKHxhthScHkjGCh1664014150693");
    // TODO: 模拟登录获取sessionId
    pageMap.put("sessionId", "16f6a3fe2b344e1f9796720e38ee7e68"); // 登录获取
    pageMap.put("pageName", "supplier");
    pageMap.put("_bx-v", "1.1.20");
    Document doc = HttpUtil.getHttpDocument2(productApi, pageMap, null, null);

    String uriRegex = "\\{\"([a-zA-Z_]+)\":(.+)}";
    List<JSONObject> modelList = new ArrayList<>();
    String data = doc.body().toString();
    // 分页查询，每页数据分两次访问
    for (int i = 1; i < pageSize; i++) {
      pageMap.put("beginPage", i);
      for (int start = 0; start < 2; start++) {
        pageMap.put("startIndex", 10 * start);
        for (String uri : JsonUtil.getJsonResult(doc.body().toString(), uriRegex)) {
          JSONObject response = JSON.parseObject(uri);
          JSONObject resData = JSON.parseObject(response.get("data").toString());
          JSONObject companyData = JSON.parseObject(resData.get("data").toString());
          JSONArray jsonArray = JSON.parseArray(
              companyData.get("companyWithOfferLists").toString());
          for (int j = 0; j < jsonArray.size(); j++) {
            JSONObject model = jsonArray.getJSONObject(j).getJSONObject("companyModel");
            modelList.add(model);
          }
        }
      }
    }

    System.out.println("=========完成所有商铺信息导出===========");
  }

  private static void getAllCompany() throws Exception {
    Document document = HttpUtil.getHttpDocument(HttpUtil.COMPANY_URL, null, null, null);
    // "totalCount":6642
    String dataRegex = "window.data.offerresultData = successDataCheck(.+)";
    List<String> jsonResult = JsonUtil.getJsonResult(document.data(), dataRegex);

    // 获取最大页码
    String regex = "\"pageCount\":(\\d+)";
    List<String> countRes = JsonUtil.getJsonResult(jsonResult.get(0), regex);
    String[] countData = countRes.get(0).split(":");
    int pageSize = Integer.parseInt(countData[1]);

    List<JSONObject> modelList = new ArrayList<>();
    for (int i = 1; i <= pageSize; i++) {
      Map<String, Object> pageMap = new HashMap<>();
      pageMap.put("beginPage", i);
      pageMap.put("async", false);
      //      pageMap.put("asyncCount", 6);
      // "domainUri":"shop1394557011429.1688.com"
      Document doc = HttpUtil.getHttpDocument(HttpUtil.COMPANY_URL, pageMap, null, null);
      // 解析出响应数据对象
      List<String> jsonRes = JsonUtil.getJsonResult(doc.data(), dataRegex);
      String uriRegex = "\\{\"([a-zA-Z_]+)\":(.+)}";
      for (String uri : JsonUtil.getJsonResult(jsonRes.get(0), uriRegex)) {
        // data.companyWithOfferLists[index].companyModel.userId/domainUri
        JSONObject response = JSON.parseObject(uri);
        JSONObject resData = JSON.parseObject(response.get("data").toString());
        JSONArray jsonArray = JSON.parseArray(resData.get("companyWithOfferLists").toString());
        for (int j = 0; j < jsonArray.size(); j++) {
          JSONObject model = jsonArray.getJSONObject(j).getJSONObject("companyModel");
          modelList.add(model);
        }
      }
    }

    // 循环商铺链接 id.1688.com，获取工厂档案和联系方式
    for (JSONObject model : modelList) {
      Map<String, Object> map = new HashMap<>();
      // 获取工厂档案 https://sale.1688.com/factory/card.html?memberId=b2b-221361970300176b32
      map.put("memberId", model.get("userId"));
      Document fileDoc = HttpUtil.getHttpDocument(HttpUtil.COMPANY_FILE_URL, map, null, null);
      // 获取联系方式 https://shop62m741s296s29.1688.com/page/contactinfo.html
      Document contactDoc = HttpUtil.getHttpDocument(
          HttpUtil.HTTP_PREFIX + model.get("domainUri") + "/page/contactinfo.html");
    }
    // 将信息存储到 company-info.xlsx
  }

  private static void selectCompany(String keywords, String province, String city)
      throws Exception {
    Map<String, Object> params = new HashMap<>();
    params.put("keywords", "");
    // 可以不加spm参数
    params.put("spm", "a26352.13672862.searchbox.input");
    // 按省市条件过滤，自动添加参数filt=y
    params.put("filt", "y");
    params.put("province", "");
    params.put("city", "");
    Document productInfo = HttpUtil.getHttpDocument(HttpUtil.COMPANY_URL, params, null, null);
    // 将查询结果保存到 company-search.xlsx
  }

}
